import yaml
from spiders.IreadSpider import IreadSpider
from util.MySqlOperate import MySqlOperate
from util.EsOperate import EsOperate

HEADER = {
    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36"}


class Main:
    def __init__(self):
        # 加载yml文件中的信息
        file = open('./resources/config.yml', 'r', encoding='utf-8')
        yml_data = file.read()
        datas = yaml.load(yml_data, Loader=yaml.FullLoader)
        file.close()
        self.mySqlOperate = MySqlOperate(datas['mysql']['maxconnections'], datas['mysql']['mincached'],
                                         datas['mysql']['maxcached'], datas['mysql']['maxusage'],
                                         datas['mysql']['host'], datas['mysql']['port'],
                                         datas['mysql']['user'], datas['mysql']['passwd'],
                                         datas['mysql']['db'], datas['mysql']['charset'])
        self.esOperate = EsOperate(datas['elasticsearch']['url'], datas['elasticsearch']['index'])
        self.page = datas['url']['indexpage']
        self.maxpage = datas['url']['maxpage']

    def main(self):
        # TODO 爬取文档
        ireadSpider = IreadSpider(HEADER, self.mySqlOperate, self.esOperate)
        while self.page < self.maxpage:
            ireadSpider.run(self.page)
            self.page += 1
        # 写回已经爬取的页数
        file = open('./resources/config.yml', 'r', encoding='utf-8')
        yml_data = file.read()
        datas = yaml.load(yml_data, Loader=yaml.FullLoader)
        file.close()
        datas['url']['indexpage'] = self.page
        file = open('./resources/config.yml', 'w', encoding='utf-8')
        yaml.dump(datas, file)
        file.close()


if __name__ == '__main__':
    Main().main()
